#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# 还原短url，将其保存到RegularUrl.txt
# 从还原后的url中抽取host字段，保存到host_from_short.txt
# Author：FWH

from urllib import request,error
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool
import datetime
from urllib.parse import urlparse #python3 only

def GetUrl(str):
    try:
        fp = request.urlopen(str, timeout = 5)
        url = fp.geturl()
        return url
    except:
        return ''

try:
    shorturl = open("E:\Hosts\\xuanwulab\ShortUrl.txt","r").readlines()

    pool = ThreadPool(8)  # 不设置的话，默认当前计算机的内核数

    urllist = list(pool.map(GetUrl,shorturl)) #python3中map函数若要得到列表，需使用list()转换
    pool.close()
    pool.join()

    hostlist=[]
    file_url = open("E:\Hosts\\xuanwulab\RegularUrl.txt", "w")
    for u in urllist :
        #从转换完成的正常url中，获取host
        hostname = urlparse(u).netloc
        hostlist.append(hostname)
        #将转换完成的正常url保存到文件中
        file_url.write(u + '\n')
    file_url.close()

    file_host = open("E:\Hosts\\xuanwulab\host_from_short.txt", "w")
    for host in hostlist:
        file_host.write(host+'\n')
    file_host.close()

except Exception as e:
    print(e)

